<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Retiring Data | Elasticsearch: The Definitive Guide [2.x] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch: The Definitive Guide [2.x]">
<link rel="up" href="scale.html" title="Designing for Scale">
<link rel="prev" href="index-templates.html" title="Index Templates">
<link rel="next" href="user-based.html" title="User-Based Data">
<meta name="DC.type" content="Learn/Docs/Legacy/Elasticsearch/Definitive Guide/2.x">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="2.x">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<p>
  <strong>WARNING</strong>: The 2.x versions of Elasticsearch have passed their
  <a href="https://www.elastic.co/support/eol">EOL dates</a>. If you are running
  a 2.x version, we strongly advise you to upgrade.
</p>
<p>
  This documentation is no longer maintained and may be removed. For the latest
  information, see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html">current
  Elasticsearch documentation</a>.
</p>
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch: The Definitive Guide [2.x]</a></span>
»
<span class="breadcrumb-link"><a href="modeling-your-data.html">Modeling Your Data</a></span>
»
<span class="breadcrumb-link"><a href="scale.html">Designing for Scale</a></span>
»
<span class="breadcrumb-node">Retiring Data</span>
</div>
<div class="navheader">
<span class="prev">
<a href="index-templates.html">« Index Templates</a>
</span>
<span class="next">
<a href="user-based.html">User-Based Data »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="retiring-data"></a>Retiring Data<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/410_Scaling/55_Retiring_data.asciidoc">edit</a>
</h2>
</div></div></div>
<p>As time-based data ages, it becomes less relevant.  It’s possible that we
will want to see what happened last week, last month, or even last year, but
for the most part, we’re interested in only the here and now.</p>
<p>The nice thing about an index per time frame is that it enables us to easily
delete old data: just delete the indices that are no longer relevant:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">DELETE /logs_2013*</pre>
</div>
<p>Deleting a whole index is much more efficient than deleting individual
documents: Elasticsearch just removes whole directories.</p>
<p>But deleting an index is very <em>final</em>.  There are a number of things we can
do to help data age gracefully, before we decide to delete it completely.</p>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="migrate-indices"></a>Migrate Old Indices<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/410_Scaling/55_Retiring_data.asciidoc">edit</a>
</h3>
</div></div></div>
<p>With logging data, there is likely to be one <em>hot</em> index—​the index for
today.  All new documents will be added to that index, and almost all queries
will target that index.  It should use your best hardware.</p>
<p>How does Elasticsearch know which servers are your best servers? You tell it,
by assigning arbitrary tags to each server.  For instance, you could start a
node as follows:</p>
<pre class="literallayout">./bin/elasticsearch --node.box_type strong</pre>

<p>The <code class="literal">box_type</code> parameter is completely arbitrary—​you could have named it
whatever you like—​but you can use these arbitrary values to tell
Elasticsearch where to allocate an index.</p>
<p>We can ensure that today’s index is on our strongest boxes by creating it with
the following settings:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">PUT /logs_2014-10-01
{
  "settings": {
    "index.routing.allocation.include.box_type" : "strong"
  }
}</pre>
</div>
<p>Yesterday’s index no longer needs to be on our strongest boxes, so we can move
it to the nodes tagged as <code class="literal">medium</code> by updating its index settings:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">POST /logs_2014-09-30/_settings
{
  "index.routing.allocation.include.box_type" : "medium"
}</pre>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="optimize-indices"></a>Optimize Indices<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/410_Scaling/55_Retiring_data.asciidoc">edit</a>
</h3>
</div></div></div>
<p>Yesterday’s index is unlikely to change.  Log events are static: what
happened in the past stays in the past.  If we merge each shard down to just a
single segment, it’ll use fewer resources and will be quicker to query. We
can do this with the <a class="xref" href="merge-process.html#optimize-api" title="optimize API">optimize API</a>.</p>
<p>It would be a bad idea to optimize the index while it was still allocated to
the <code class="literal">strong</code> boxes, as the optimization process could swamp the I/O on those
nodes and impact the indexing of today’s logs.  But the <code class="literal">medium</code> boxes aren’t
doing very much at all, so we are safe to optimize.</p>
<p>Yesterday’s index may have replica shards. If we issue an optimize request, it
will optimize the primary shard and the replica shards, which is a waste.
Instead, we can remove the replicas temporarily, optimize, and then restore the
replicas:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">POST /logs_2014-09-30/_settings
{ "number_of_replicas": 0 }

POST /logs_2014-09-30/_optimize?max_num_segments=1

POST /logs_2014-09-30/_settings
{ "number_of_replicas": 1 }</pre>
</div>
<p>Of course, without replicas, we run the risk of losing data if a disk suffers
catastrophic failure.  You may want to back up the data first, with the
<a href="/guide/en/elasticsearch/reference/2.4/modules-snapshots.html" class="ulink" target="_top"><code class="literal">snapshot-restore</code> API</a>.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="close-indices"></a>Closing Old Indices<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/410_Scaling/55_Retiring_data.asciidoc">edit</a>
</h3>
</div></div></div>
<p>As indices get even older, they reach a point where they are almost never
accessed.  We could delete them at this stage, but perhaps you want to keep
them around just in case somebody asks for them in six months.</p>
<p>These indices can be closed. They will still exist in the cluster, but they
won’t consume resources other than disk space.  Reopening an index is much
quicker than restoring it from backup.</p>
<p>Before closing, it is worth flushing the index to make sure that there are no
transactions left in the transaction log.  An empty transaction log will make
index recovery faster when it is reopened:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">POST /logs_2014-01-*/_flush <a id="CO293-1"></a><i class="conum" data-value="1"></i>
POST /logs_2014-01-*/_close <a id="CO293-2"></a><i class="conum" data-value="2"></i>
POST /logs_2014-01-*/_open <a id="CO293-3"></a><i class="conum" data-value="3"></i></pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO293-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Flush all indices from January to empty the transaction logs.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO293-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>Close all indices from January.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO293-3"><i class="conum" data-value="3"></i></a></p>
</td>
<td align="left" valign="top">
<p>When you need access to them again, reopen them with the <code class="literal">open</code> API.</p>
</td>
</tr>
</table>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="archive-indices"></a>Archiving Old Indices<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/410_Scaling/55_Retiring_data.asciidoc">edit</a>
</h3>
</div></div></div>
<p>Finally, very old indices can be archived off to some long-term storage like a
shared disk or Amazon’s S3 using the
<a href="/guide/en/elasticsearch/reference/2.4/modules-snapshots.html" class="ulink" target="_top"><code class="literal">snapshot-restore</code> API</a>, just in case you may need
to access them in the future.  Once a backup exists, the index can be deleted
from the cluster.</p>
</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="index-templates.html">« Index Templates</a>
</span>
<span class="next">
<a href="user-based.html">User-Based Data »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
